import download from "./utils/download.js";
import { promiseLimit } from "./utils/promise.js";
import config, { saveConfig } from "./config.js";
import { glob } from "glob";
import fse from "fs-extra";

async function main() {
  let files = await glob("./data/*.json").then((res) =>
    res.sort((a, b) => {
      return +a.match(/\d+/) - +b.match(/\d+/);
    })
  );

  for (let i = config.page; i < files.length; i++) {
    config.page = i;

    const data = fse.readJsonSync(files[i], { throws: false });

    await downloadAllPdfs(data.data);

    config.pageIndex = 0;
  }

  console.log("结束了");
}

async function downloadAllPdfs(list) {
  for (let i = config.pageIndex; i < list.length; i++) {
    const item = list[i];
    const index = i;

    console.log(
      `进度:${config.page}页 ${index + 1}/${list.length} (${Math.round(
        ((index + 1) / list.length) * 100
      )}%)`
    );

    await downloadPdf(item);

    // 等个3-10秒
    await new Promise((resolve) => setTimeout(resolve, 30 * 1000));

    config.pageIndex = index;
    saveConfig();
  }
}

async function downloadPdf(item) {
  console.log("downloadPdf");
  const page = config.page + 1;

  const match = item.download.match(/href="(\/download\/\d+)"/);
  const downloadPath = "https://www.exploit-db.com" + match[1]; // "/download/50552"
  console.log(downloadPath); // 输出: /download/50552
  await download(downloadPath, `output/${page}`).catch((e) => {
    console.error(e);

    if (!config.errors[page]) config.errors[page] = {};

    config.errors[page][downloadPath] = true;
    saveConfig();
  });
  console.log("下载完成");
}

main();
